﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml.Linq;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;

namespace WindowsPhone.Marketplace
{
    public class Crawler
    {
        private MarketplaceSettings _settings;

        public Crawler(MarketplaceSettings settings = null)
        {
            _settings = settings ?? new MarketplaceSettings();
        }

        /// <summary>
        /// Dumps all of the apps for a given language to a file
        /// </summary>
        /// <param name="lang"></param>
        public void GetAllApps(string path, string lang = "en-US")
        {
            string xml;
            string nextUrl = null;

            bool keepXmlHeader = true;

            int fileNum = 1;
            string fileName = null;
            FileInfo fileInfo;

            StreamWriter writer = null;

            do
            {

                if (fileName != null)
                {
                    fileInfo = new FileInfo(fileName);

                    if (fileInfo.Length > 100000000)
                    {
                        fileNum++;
                        fileName = null;
                        
                        // close the current file
                        writer.Write("</a:feed>");
                        writer.Dispose();

                        keepXmlHeader = true;
                    }
                }

                if (string.IsNullOrEmpty(fileName))
                {
                    fileName = Path.Combine(path, "marketplace_") + lang + "_" + fileNum + ".xml";

                    // left over files from an old run? delete them.
                    if (File.Exists(fileName))
                        File.Delete(fileName);

                    writer = new StreamWriter(fileName);
                }

                xml = GetAppsXml(nextUrl, lang);
                nextUrl = GetNextAppsUrl(xml);

                // clear out the common bits so that we get one large XMl document
                if (!keepXmlHeader)
                {
                    xml = CleanXmlHeaderFields(xml);
                }
                else
                {
                    keepXmlHeader = false;
                }

                if (xml.EndsWith("</a:feed>"))
                    xml = xml.Substring(0, xml.Length - "</a:feed>".Length);

                if (!string.IsNullOrEmpty(xml))
                    writer.Write(xml + "\n");

                if (nextUrl == null)
                {
                    writer.Write("</a:feed>");
                    writer.Dispose();

                    xml = null;
                }

                Console.WriteLine(nextUrl);

            } while (!string.IsNullOrEmpty(xml));
            
        }

        private string CleanXmlHeaderFields(string xml)
        {
            Regex re = new Regex(@"(<\?xml[\S\s]*?)<a:entry");

            Match match = re.Match(xml);

            if (match.Groups.Count > 1)
            {
                xml = xml.Replace(match.Groups[1].Value, "");
            }
            else
            {
                // most likely empty XML
                if (xml.Contains("entry"))
                {
                    System.Diagnostics.Debugger.Break();
                }
                else
                {
                    xml = ""; // this is the footer XML, so clear it out since we don't need it
                }
            }

            return xml;
        }

        public string GetAppsXml(string url, string lang)
        {
            string fullUrl;
            bool done = false;
            string rv = null;

            fullUrl = (!String.IsNullOrEmpty(url) ? url : _settings.BaseAppsUrl + _settings.ZestVersion + lang + "/" + _settings.BaseApps + _settings.BaseAppsResource);

            while (!done)
            {
                try
                {
                    var request = WebRequest.Create(fullUrl) as HttpWebRequest;
                    request.KeepAlive = false;

                    var response = request.GetResponse() as HttpWebResponse;

                    if (request.HaveResponse == true && response != null)
                    {
                        var reader = new StreamReader(response.GetResponseStream());
                        rv = reader.ReadToEnd();
                        done = true;
                    }
                }
                catch
                {
                    Console.WriteLine("yeah, your connection was likely aborted");
                    done = false;
                }
            }

            return rv;
        }

        public string GetNextAppsUrl(string rawXml)
        {
            XElement xml = XElement.Parse(rawXml);

            //now I need to get the AfterMarkerUrl from the XML feed
            var afterMarker =
                from e in xml.Elements(MarketplaceSettings.AtomNamespace + "link")
                where e.Attribute("rel").Value == "next"
                select (string)e.Attribute("href").Value;


            return (afterMarker.Count() > 0 ? _settings.BaseAppsUrl + afterMarker.Single() : null);
        }
    }
}
